'use strict';

const {chromium} = require('playwright');

const Service = require('egg').Service;

const headless = true;

const proxy_obj = {
  server: 'http://u5329.b5.t.16yun.cn:6460',
  username: '16GADNHJ',
  password: '936441'
};

class CrawlerService extends Service {

  async fetchTopics(author_name, author_id, author_avatar, url, tag) {

    const cache_key = 'tophub:' + author_name + ':' + tag;
    let exist_titles = await this.service.cacheSvc.get(cache_key);
    if (!exist_titles) {
      exist_titles = [];
    }
    let new_titles = await this.getAllTitles(url, 2);
    new_titles = new_titles.slice(0, 100);

    let need_fetch_idxs = [];
    for (let i = 0; i < new_titles.length; i++) {
      let title = new_titles[i];
      let isExist = false;
      for (const v of exist_titles) {
        if (v == title) {
          isExist = true;
          break;
        }
      }
      if (!isExist) {
        need_fetch_idxs.push(i);
      }
    }

    let idx_array2 = [];
    for (let i = 0; i < need_fetch_idxs.length; i = i + 10) {
      idx_array2.push(need_fetch_idxs.slice(i, i + 10));
    }

    console.log('tophub 本次新变化的个数:', need_fetch_idxs.length, author_name, tag);
    this.logger.info('tophub 本次新变化的个数:', need_fetch_idxs.length, author_name, tag);

    let totalTopics = [];

    for (let i = 0; i < idx_array2.length; i++) {
      let r = await this.fetchTopicsFragments(
        author_name,
        author_id,
        author_avatar,
        url,
        tag,
        idx_array2[i],
        2
      );
      totalTopics = totalTopics.concat(r);
    }

    await this.service.cacheSvc.setex(cache_key, new_titles);

    if (totalTopics.length > 0) {
      this.logger.info(author_name, tag, 'success...');
    }
    return totalTopics;
  }

  async getAllTitles(url, idx_tab) {
    const browser = await chromium.launch({headless: headless});
    let allTitles;
    try {
      let context;
      let page;
      while (page == null) {
        try {
          [context, page] = await this.getCanUseContextAndPage(browser, url);
        } catch (e) {
          this.logger.warn('重新尝试1...');
        }
      }
      await page.waitForSelector(`div:nth-child(${idx_tab}) > div > div > div > table > tbody > tr`);
      allTitles = await page.$$eval(`#page > div > div > div:nth-child(${idx_tab}) > div > div > div > table > tbody > tr > td.al > a`, vs => vs.map(v => v.innerText));
    } finally {
      await browser.close();
      return allTitles;
    }
  }

  async fetchTopicsFragments(author_name, author_id, author_avatar, url, tag, idxs, idx_tab) {
    let totalTopics = [];
    let context;
    let page;
    const sub_tab = author_name;
    const source = sub_tab + (new Date(+new Date() + 1000 * 3600 * 8)).toISOString().split('T')[0];

    let browser = await chromium.launch({headless: headless});
    try {
      while (page == null) {
        try {
          [context, page] = await this.getCanUseContextAndPage(browser, url);
        } catch (e) {
          this.logger.warn('重新尝试2...');
        }
      }

      await page.waitForSelector(`div:nth-child(${idx_tab}) > div > div > div > table > tbody > tr`);

      for (let i = 0; i < idxs.length; i++) {
        let newPage;

        while (page == null) {
          try {
            [context, page] = await this.getCanUseContextAndPage(browser, url);
          } catch (e) {
            this.logger.warn('重新尝试3...');
          }
        }


        try {
          let pointer = idxs[i];
          let topic = {};
          let idx = `#page > div > div > div:nth-child(${idx_tab}) > div > div > div > table > tbody > tr:nth-child(${pointer + 1}) > td.al > a`;

          await page.waitForSelector(`div:nth-child(${idx_tab}) > div > div > div > table > tbody > tr`);

          topic.title = await page.$eval(idx, el => el.innerText);

          [newPage] = await Promise.all([
            context.waitForEvent('page', {timeout: 10000}),
            page.click(idx),
          ]);

          let original_url = newPage.url();

          if (original_url.startsWith('https://passport.weibo.com')) {
            original_url = decodeURIComponent(original_url);
            original_url = original_url
              .replace(/.+(https:\/\/s.weibo.com.+&Refer=top).+/, '$1')
              .replace(/.+(https:\/\/s.weibo.com.+)&domain.+/, '$1')
              .replace(/.+(https:\/\/weibo.com.+)\?from=.+/, '$1')
              .replace('/tv/tv/', '/tv/');
          }

          topic.content = original_url;
          topic.original_url = original_url;
          topic.tags = [tag];
          topic.tab = 'hot';
          topic.sub_tab = sub_tab;
          topic.source = source;
          topic.author_id = author_id;
          topic.author_name = author_name;
          topic.author_avatar = author_avatar;

          if (topic.content.includes('chrome-error://')) {
            throw new Error('url错误:' + topic.title);
          }

          if (topic.content.includes('http://cli.tophub.today/tool/ip.unlock.php')) {
            throw new Error('url错误:' + topic.title);
          }

          console.log('success:', topic.content, topic.title, author_name, tag, pointer);
          totalTopics.push(topic);

        } catch (e) {
          console.error('xxxxx', e);
          await page.close();
          await context.close();
          page = null;
          context = null;

          // 代表后面没有内容了
          if (e.message.includes('failed to find element matching selector')) {
            break;
          }

          i--;
          continue;
        } finally {
          if (newPage) {
            await newPage.close();
          }
        }
      }

    } catch (e) {
      console.error(e);
      this.logger.error(e);
    } finally {
      await browser.close();
      if (totalTopics.length == 0) {
        this.logger.error(author_name, '爬取异常!!!');
      }
      return totalTopics;
    }
  }


  async getCanUseContextAndPage(browser, url) {
    let context = await browser.newContext({proxy: proxy_obj});
    let page = await context.newPage();
    await page.setExtraHTTPHeaders({'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.70 Safari/537.36'});
    await page.goto(url, {waitUntil: 'domcontentloaded', timeout: 10000});

    let now_url = await page.url();
    while (now_url == 'http://cli.tophub.today/tool/ip.unlock.php') {
      await context.close();
      context = await browser.newContext({proxy: proxy_obj});
      page = await context.newPage();
      await page.goto(url, {waitUntil: 'domcontentloaded'});
      now_url = await page.url();
    }
    return [context, page];
  }
}

module.exports = CrawlerService;
